/**
 * (created at 2010-12-27)
 */
package qiushuo.mysqlparser.recognizer;

import static qiushuo.mysqlparser.recognizer.SQLTokenType.EOF;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.IDENTIFIER;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.LITERAL_BIT_FIELD;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.LITERAL_BOOL_FALSE;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.LITERAL_BOOL_TRUE;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.LITERAL_HEXADECIMAL;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.LITERAL_NULL;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.LITERAL_NUMBER;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.LITERAL_STRING;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OPERATOR;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_AMPERSAND;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_ASSIGN;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_ASTERISK;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_CARET;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_EQUALS;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_EXCLAMATION;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_GREATER_OR_EQUALS;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_GREATER_THAN;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_LEFT_SHIFT;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_LESS_OR_EQUALS;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_LESS_OR_GREATER;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_LESS_THAN;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_LOGICAL_AND;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_LOGICAL_OR;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_MINUS;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_NOT_EQUALS;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_NULL_SAFE_EQUALS;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_PERCENT;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_PLUS;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_RIGHT_SHIFT;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_SLASH;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_TILDE;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.OP_VERTICAL_BAR;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNCTUATION;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNC_COLON;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNC_COMMA;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNC_LEFT_BRACE;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNC_LEFT_BRACKET;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNC_LEFT_PAREN;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNC_OTHER;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNC_RIGHT_BRACE;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNC_RIGHT_BRACKET;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNC_RIGHT_PAREN;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.PUNC_SEMICOLON;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.SC_QUESTION;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.SPECIAL_CHAR;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.SYSTEM_VARIABLE;
import static qiushuo.mysqlparser.recognizer.SQLTokenType.USER_DEFINED_VARIABLE;

import java.util.HashMap;
import java.util.Map;

/**
 * @author <a href="mailto:QiuShuo1985@gmail.com">QIU Shuo</a>
 */
public class SQLLexer {
    private CharReader reader;

    public SQLLexer(CharReader reader) {
        this.reader = reader;
    }

    /**
     * assumption: seperator has been skipped
     */
    private SQLTokenType makeDecision() {
        if (!reader.hasNext()) return EOF;
        char c = reader.lookAhead(0);
        switch (c) {
        case '0':
            if (reader.hasNext(2)) {
                switch (reader.lookAhead(1)) {
                case 'b':
                    return LITERAL_BIT_FIELD;
                case 'x':
                    return LITERAL_HEXADECIMAL;
                default:
                    return LITERAL_NUMBER;
                }
            }
            return LITERAL_NUMBER;
        case 'x':
        case 'X':
            if (reader.hasNext(2)) {
                switch (reader.lookAhead(1)) {
                case '\'':
                    return LITERAL_HEXADECIMAL;
                default:
                    return IDENTIFIER;
                }
            }
            return IDENTIFIER;
        case 'b':
        case 'B':
            if (reader.hasNext(2)) {
                switch (reader.lookAhead(1)) {
                case '\'':
                    return LITERAL_BIT_FIELD;
                default:
                    return IDENTIFIER;
                }
            }
            return IDENTIFIER;
        case 'n':
        case 'N':
            if (reader.hasNext(2)) {
                switch (reader.lookAhead(1)) {
                case '\'':
                    return LITERAL_STRING;
                default:
                    return IDENTIFIER;
                }
            }
            return IDENTIFIER;
        case '=':
        case '~':
        case '*':
        case '-':
        case '+':
        case '^':
        case '/':
        case '%':
        case '&':
        case '|':
        case '!':
        case '>':
        case '<':
            return OPERATOR;
        case '@':
            if (reader.hasNext(2) && reader.lookAhead(1) == '@') {
                return SYSTEM_VARIABLE;
            }
            return USER_DEFINED_VARIABLE;
        case '\'':
        case '"':
            return LITERAL_STRING;
        case '?':
            return SPECIAL_CHAR;
        case '.':
            if (reader.hasNext(2) && isDigit(reader.lookAhead(1))) return LITERAL_NUMBER;
            return IDENTIFIER;
        case ';':
        case ',':
        case '(':
        case ')':
        case '[':
        case ']':
        case '{':
        case '}':
            return PUNCTUATION;
        case ':':
            if (reader.hasNext(2) && reader.lookAhead(1) == '=') return OPERATOR;
            return PUNCTUATION;
        case '`':
        case '$':
        case '_':
            return IDENTIFIER;
        default:
            if (isDigit(c)) return LITERAL_NUMBER;
            if (Character.isLetterOrDigit(c)) return IDENTIFIER;
            return PUNCTUATION;
        }
    }

    /**
     * @return "n'abcde'" or "_utf8'ab\\'cde'"
     */
    private SQLToken matchLiteralString() {
        final int pos = reader.getCurrentPosition();
        if (!reader.hasNext()) return new SQLToken(null, pos, 0, EOF);
        reader.mark();
        StringBuilder sb = new StringBuilder();
        char c = reader.next();
        // deal with prefix
        switch (c) {
        case '\'':
        case '"':
            reader.backForce();
            break;
        case 'n':
        case 'N':
            sb.append('n');
            if (reader.lookAhead(0) != '\'') err("expect a '\\'' after 'n' for string literal");
            break;
        case '_':
            sb.append('_');
            loop1: for (int i = 0;; ++i) {
                c = reader.next();
                switch (c) {
                case '\'':
                case '"':
                    if (i <= 0) err("must have charset_name after '_' for string literal");
                    reader.backForce();
                    break loop1;
                case '_':
                case '$':
                    sb.append(c);
                    continue;
                default:
                    if (Character.isLetterOrDigit(c)) {
                        sb.append(c);
                        continue;
                    }
                    err("invalid char for string prefix: " + c);
                    break;
                }
            }
            break;
        default:
            err("invalid char for string prefix: " + c);
        }
        sb.append('\'');
        h_readQuotedString(sb);
        while (true) {
            skipSeparator();
            if (reader.hasNext()) {
                if ((reader.lookAhead(0) == '\'' || reader.lookAhead(0) == '"')) {
                    h_readQuotedString(sb);
                    continue;
                }
            }
            break;
        }
        sb.append('\'');
        return new SQLToken(sb.toString(), pos, reader.getCurrentPosition() - pos, LITERAL_STRING);
    }

    /**
     * trim first and second '\'', e.g. for "'don\\'t'", return "don\\'t"
     */
    private void h_readQuotedString(StringBuilder sb) {
        boolean doubleQ = false;
        char c = reader.next();
        switch (c) {
        case '\'':
            break;
        case '"':
            doubleQ = true;
            break;
        default:
            err("expecting '\\'' or '\"' for string literal");
        }
        while (true) {
            c = reader.next();
            switch (c) {
            case '\\':
                c = reader.next();
                switch (c) {
                case '0':
                case '\'':
                case 'b':
                case 'n':
                case 'r':
                case 't':
                case 'Z':
                case '\\':
                case '%':
                case '_':
                    sb.append('\\');
                }
                sb.append(c);
                break;
            case '"':
                if (doubleQ) {
                    if (reader.hasNext() && reader.lookAhead(0) == '"') {
                        sb.append(reader.next());
                        continue;
                    }
                    return;
                } else {
                    sb.append('"');
                }
                break;
            case '\'':
                if (doubleQ) {
                    sb.append("\\'");
                } else {
                    if (reader.hasNext() && reader.lookAhead(0) == '\'') {
                        sb.append('\\').append(reader.next());
                        continue;
                    }
                    return;
                }
                break;
            default:
                sb.append(c);
            }
        }
    }

    /**
     * @param true if e.g. "_latin1 0x9YUI"
     */
    private SQLToken matchIdentifier(boolean fromHexBit) {
        int pos = reader.getCurrentPosition();
        StringBuilder sb = new StringBuilder();
        reader.mark();
        if ('_' == reader.lookAhead(0)) {
            h_readUntillEndChar(sb, true);
            if (reader.hasNext()) {
                switch (reader.lookAhead(0)) {
                case '.':
                    break;
                case '\'':
                case '"':
                    reader.reset();
                    return matchLiteralString();
                default:
                    if (isSeparator()) {
                        skipSeparator();
                        if (!reader.isNext(".")) {
                            if (!fromHexBit) {
                                if (reader.isNext("0x") || reader.isNext("x'") || reader.isNext("X'")) {
                                    reader.reset();
                                    return matchLiteralHex();
                                } else if (reader.isNext("0b") || reader.isNext("b'") || reader.isNext("B'")) {
                                    reader.reset();
                                    return matchLiteralBit();
                                }
                            }
                            return new SQLToken(sb.toString(), pos, reader.getCurrentPosition() - pos, IDENTIFIER);
                        }
                    } else {
                        return new SQLToken(sb.toString(), pos, reader.getCurrentPosition() - pos, IDENTIFIER);
                    }
                }
            } else {
                return new SQLToken(sb.toString(), pos, reader.getCurrentPosition() - pos, IDENTIFIER);
            }
        }
        if (reader.isNext(".")) {
            sb.append(reader.next());
            skipSeparator();
        }
        while (h_readIdentifierPart(sb)) {
            sb.append(reader.next());// '.'
            skipSeparator();
        }
        if (sb.length() == 0) err("empty id: " + sb);
        return new SQLToken(sb.toString(), pos, reader.getCurrentPosition() - pos, IDENTIFIER);
    }

    /**
     * separator already skipped before this method is called. after invocation,
     * move reader to '.' if there is one
     * 
     * @return is '.' after part
     */
    private boolean h_readIdentifierPart(StringBuilder sb) {
        if (!reader.hasNext()) err("EOF after '.' or EOF at the beginning of id");
        int initLen = sb.length();
        char c = reader.lookAhead(0);
        switch (c) {
        case '`':
            sb.append(reader.next());
            while (true) {
                if (!reader.hasNext()) err("no closing '`' after openning '`' for id");
                c = reader.next();
                if (c == '`') {
                    sb.append('`');
                    if (reader.hasNext() && reader.lookAhead(0) == '`') {
                        sb.append(reader.next());
                    } else {
                        break;
                    }
                } else {
                    sb.append(c);
                }
            }
            break;
        case '*':
            sb.append(reader.next());
            if (reader.hasNext()) {
                c = reader.lookAhead(0);
                if (!isEndCharacter(c)) err();
            }
            break;
        default:
            h_readUntillEndChar(sb, true);
        }

        if (sb.length() == initLen) err("encounter empty id part");
        skipSeparator();
        return reader.hasNext() && reader.lookAhead(0) == '.';
    }

    private void h_readUntillEndChar(StringBuilder sb, boolean stopAtDot) {
        if (stopAtDot) {
            while (reader.hasNext()) {
                char c = reader.next();
                if (isEndCharacter(c) || c == '.') {
                    reader.backForce();
                    break;
                }
                sb.append(c);
            }
        } else {
            while (reader.hasNext()) {
                char c = reader.next();
                if (isEndCharacter(c)) {
                    reader.backForce();
                    break;
                }
                sb.append(c);
            }
        }
    }

    private SQLToken matchLiteralHex() {
        final int pos = reader.getCurrentPosition();
        if (!reader.hasNext()) return new SQLToken(null, pos, 0, EOF);
        StringBuilder sb = new StringBuilder();
        reader.mark();
        if ('_' == reader.lookAhead(0)) {
            h_matchCharsetIntroducer4HexOrBit(sb);
        }
        char c = reader.next();
        switch (c) {
        case '0':
            try {
                if (reader.next() != 'x') err();
            } catch (StringIndexOutOfBoundsException e) {
                return new SQLToken("0", pos, 1, LITERAL_NUMBER);
            }
            while (reader.hasNext()) {
                c = reader.next();
                if (isHexDigit(c)) {
                    sb.append(Character.toLowerCase(c));
                } else if (isEndCharacter(c)) {
                    reader.backForce();
                    break;
                } else {
                    reader.reset();
                    return matchIdentifier(true);
                }
            }
            break;
        case 'x':
        case 'X':
            if (reader.next() != '\'') err();
            while (true) {
                c = reader.next();
                if (isHexDigit(c)) {
                    sb.append(Character.toLowerCase(c));
                } else if (c == '\'') {
                    break;
                } else {
                    err("invalid char: " + c);
                }
            }
            break;
        default:
            err();
        }
        int insertIndx = 0;
        if (sb.length() > 0 && sb.charAt(0) == '_') {
            insertIndx = sb.indexOf(" ") + 1;
        }
        if (sb.length() == 0) {
            sb.append("00");
        } else if (sb.length() % 2 != 0) {
            sb.insert(insertIndx, '0');
        }
        String text = sb.insert(insertIndx, "0x").toString();
        return new SQLToken(text, pos, reader.getCurrentPosition() - pos, LITERAL_HEXADECIMAL);
    }

    private SQLToken matchLiteralBit() {
        final int pos = reader.getCurrentPosition();
        if (!reader.hasNext()) return new SQLToken(null, pos, 0, EOF);
        reader.mark();
        StringBuilder sb = new StringBuilder();
        if ('_' == reader.lookAhead(0)) {
            h_matchCharsetIntroducer4HexOrBit(sb);
        }
        char c = reader.next();
        switch (c) {
        case '0':
            try {
                if (reader.next() != 'b') err();
            } catch (StringIndexOutOfBoundsException e) {
                return new SQLToken("0", pos, 1, LITERAL_NUMBER);
            }
            loop1: while (reader.hasNext()) {
                c = reader.next();
                switch (c) {
                case '1':
                case '0':
                    sb.append(c);
                    break;
                default:
                    if (isEndCharacter(c)) {
                        reader.backForce();
                        break loop1;
                    } else {
                        reader.reset();
                        return matchIdentifier(true);
                    }
                }
            }
            break;
        case 'b':
        case 'B':
            if (reader.next() != '\'') err();
            loop1: while (true) {
                c = reader.next();
                switch (c) {
                case '1':
                case '0':
                    sb.append(c);
                    break;
                case '\'':
                    break loop1;
                default:
                    err("invalid char: " + c);
                }
            }
            break;
        default:
            err();
        }
        int insertIndx = 0;
        if (sb.length() > 0 && sb.charAt(0) == '_') {
            insertIndx = sb.indexOf(" ") + 1;
        }
        if (sb.length() == 0) {
            sb.append('0');
        }
        String text = sb.insert(insertIndx, "0b").toString();
        return new SQLToken(text, pos, reader.getCurrentPosition() - pos, LITERAL_BIT_FIELD);
    }

    private void h_matchCharsetIntroducer4HexOrBit(StringBuilder sb) {
        for (int i = 0;; ++i) {
            if (!reader.hasNext()) {
                if (i == 0) err("must have collate name");
            }
            char c = reader.next();
            if (isEndCharacter(c)) {
                if (i == 0) err("must have collate name");
                reader.backForce();
                break;
            }
            sb.append(c);
        }
        sb.append(' ');
        skipSeparator();
    }

    /**
     * e.g. "0.123e+23". never return useless '0'("00.xxx") or 'e'("xxxe+0" or
     * "0exxx") or '.'("123e+2" instead of "123.e+2"); always sign after
     * 'e'("123.2e+12" instead of "123.2e12")
     */
    private SQLToken matchLiteralNumber() {
        final int pos = reader.getCurrentPosition();
        if (!reader.hasNext()) return new SQLToken(null, pos, 0, EOF);
        reader.mark();
        StringBuilder sb = new StringBuilder();
        char c = reader.next();
        int state = 0;
        // process state 0 (before .)
        if (c == '.') {
            sb.append("0.");
            state = 1;
        } else if (isDigit(c)) {
            sb.append(c);
            loop1: while (true) {
                if (!reader.hasNext() || isEndCharacter(reader.lookAhead(0))) {
                    state = 3;
                    break;
                }
                c = reader.next();
                switch (c) {
                case '.':
                    sb.append('.');
                    state = 1;
                    break loop1;
                case 'e':
                case 'E':
                    sb.append('e');
                    state = 2;
                    break loop1;
                default:
                    if (isDigit(c)) {
                        sb.append(c);
                    } else {
                        reader.reset();
                        return matchIdentifier(false);
                    }
                }
            }
        } else {
            err("invalid char: " + c);
        }
        // process state 1 (before e, after .)
        if (state < 1) {
            throw new RuntimeException("bug: state must be greater than 0 here!");
        } else if (state == 1) {
            loop1: while (true) {
                if (!reader.hasNext() || isEndCharacter(reader.lookAhead(0))) {
                    state = 3;
                    break;
                }
                c = reader.next();
                switch (c) {
                case 'e':
                case 'E':
                    state = 2;
                    if (sb.length() > 0 && sb.charAt(sb.length() - 1) == '.') {
                        sb.deleteCharAt(sb.length() - 1);
                    }
                    sb.append('e');
                    break loop1;
                default:
                    if (isDigit(c)) {
                        sb.append(c);
                    } else {
                        reader.reset();
                        return matchIdentifier(false);
                    }
                    break;
                }
            }
        }
        // process state 2 (after e)
        if (state < 2) {
            throw new RuntimeException("bug: state must be greater than 1 here!");
        } else if (state == 2) {
            boolean signed = false;
            if (reader.hasNext()) {
                switch (reader.lookAhead(0)) {
                case '-':
                case '+':
                    signed = true;
                    sb.append(reader.next());
                    break;
                default:
                    break;
                }
            } else {
                reader.reset();
                return matchIdentifier(false);
            }
            c = reader.next();
            if (isDigit(c)) {
                if (!signed) sb.append('+');
                sb.append(c);
                while (true) {
                    if (!reader.hasNext() || isEndCharacter(reader.lookAhead(0))) {
                        state = 3;
                        break;
                    }
                    c = reader.next();
                    if (isDigit(c)) {
                        sb.append(c);
                    } else {
                        reader.reset();
                        return matchIdentifier(false);
                    }
                }
            } else {
                reader.reset();
                return matchIdentifier(false);
            }
        }
        h_normalizeNumberLiteralE(sb);
        boolean f0 = h_normalizeNumberLiteralDot(sb);
        boolean i0 = h_normalizeNumberLiteralInteger(sb);
        if (f0 && i0) {
            return new SQLToken("0", pos, reader.getCurrentPosition() - pos, LITERAL_NUMBER);
        }
        return new SQLToken(sb.toString(), pos, reader.getCurrentPosition() - pos, LITERAL_NUMBER);
    }

    /**
     * @return true 锟斤拷锟斤拷位为0
     */
    private boolean h_normalizeNumberLiteralInteger(StringBuilder sb) {
        if (sb == null || sb.length() == 0) err("bug: invalid raw token for numberLiteral: " + sb);
        int end = sb.indexOf(".");
        if (end < 0) end = sb.indexOf("e");
        if (end < 0) end = sb.length();
        boolean rst = true;
        int toremovePre = 0;
        loop1: for (int i = 0; i < end; ++i) {
            char c = sb.charAt(i);
            switch (c) {
            case '0':
                if (i + 1 < end && isDigit(sb.charAt(i + 1))) {
                    ++toremovePre;
                }
                break;
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                rst = false;
                break loop1;
            default:
                err("bug: invalid raw token for numberLiteral: " + sb);
                return true;
            }
        }
        if (toremovePre > 0) {
            sb.delete(0, toremovePre);
        }
        return rst;
    }

    /**
     * @return true 小锟斤拷位为0
     */
    private boolean h_normalizeNumberLiteralDot(StringBuilder sb) {
        int indexOfDot = sb.indexOf(".");
        if (indexOfDot < 0) return true;
        int indexOfE = sb.indexOf("e");
        if (indexOfE < 0) indexOfE = sb.length();
        if (indexOfDot + 1 == indexOfE) {
            sb.deleteCharAt(indexOfDot);
            return true;
        }
        for (int i = indexOfDot + 1; i < indexOfE; ++i) {
            char c = sb.charAt(i);
            switch (c) {
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                return false;
            case '0':
                break;
            default:
                err("bug: invalid raw token for numberLiteral: " + sb);
                break;
            }
        }
        return true;
    }

    private void h_normalizeNumberLiteralE(StringBuilder sb) {
        boolean removeE = false;
        int indexOfE = sb.indexOf("e");
        if (indexOfE >= 0) {
            removeE = true;
            loop1: for (int i = indexOfE + 1; i < sb.length();) {
                char c = sb.charAt(i);
                switch (c) {
                case '-':
                case '+':
                    ++i;
                    continue;
                case '0':
                    sb.deleteCharAt(i);
                    continue;
                default:
                    removeE = false;
                    break loop1;
                }
            }
        } else {
            return;
        }
        if (removeE) {
            sb.delete(indexOfE, sb.length());
        }
    }

    private SQLToken matchSysVar() {
        final int pos = reader.getCurrentPosition();
        if (!reader.hasNext()) return new SQLToken(null, pos, 0, EOF);
        StringBuilder sb = new StringBuilder();
        if (reader.next() != '@' || reader.next() != '@') {
            err("systemVariable must start with @@");
        }
        sb.append('@').append('@');
        h_readUntillEndChar(sb, false);
        if (sb.length() <= 2) err("must have a name for userDefVar");
        return new SQLToken(sb.toString(), pos, reader.getCurrentPosition() - pos, SYSTEM_VARIABLE);
    }

    private SQLToken matchUserDefVar() {
        final int pos = reader.getCurrentPosition();
        if (!reader.hasNext()) return new SQLToken(null, pos, 0, EOF);
        char c = reader.next();
        StringBuilder sb = new StringBuilder();
        if (c != '@') err();
        sb.append('@');
        boolean content = false;
        if (!reader.hasNext()) err("must have a name for userDefVar");
        c = reader.lookAhead(0);
        switch (c) {
        case '\'':
        case '"':
            sb.append('\'');
            h_readQuotedString(sb);
            sb.append('\'');
            break;
        case '@':
            reader.backForce();
            return matchSysVar();
        default:
            loop1: while (reader.hasNext()) {
                c = reader.next();
                switch (c) {
                case '@':
                    if (!content) {
                        sb.append('@');
                        continue;
                    } else {
                        reader.backForce();
                        break loop1;
                    }
                default:
                    if (isEndCharacter(c)) {
                        reader.backForce();
                        break loop1;
                    }
                    content = true;
                    sb.append(c);
                }
            }
        }
        if (sb.length() <= 1) err("must have a name for userDefVar");
        return new SQLToken(sb.toString(), pos, reader.getCurrentPosition() - pos, USER_DEFINED_VARIABLE);
    }

    private SQLToken matchOperator() {
        final int pos = reader.getCurrentPosition();
        if (!reader.hasNext()) return new SQLToken(null, pos, 0, EOF);
        char c = reader.next();
        switch (c) {
        case '+':
            return new SQLToken("+", pos, 1, OP_PLUS);
        case '-':
            return new SQLToken("-", pos, 1, OP_MINUS);
        case '*':
            return new SQLToken("*", pos, 1, OP_ASTERISK);
        case '/':
            return new SQLToken("/", pos, 1, OP_SLASH);
        case '=':
            return new SQLToken("=", pos, 1, OP_EQUALS);
        case '~':
            return new SQLToken("~", pos, 1, OP_TILDE);
        case '^':
            return new SQLToken("^", pos, 1, OP_CARET);
        case '%':
            return new SQLToken("%", pos, 1, OP_PERCENT);
        case ':':
            if (reader.hasNext()) {
                if (reader.next() == '=') {
                    return new SQLToken(":=", pos, 2, OP_ASSIGN);
                }
                reader.skipForce(-2);
                return matchPunctuation();
            }
            reader.backForce();
            return matchPunctuation();
        case '<':
            if (reader.hasNext()) {
                switch (reader.next()) {
                case '<':
                    return new SQLToken("<<", pos, 2, OP_LEFT_SHIFT);
                case '=':
                    if (reader.hasNext()) {
                        if (reader.next() == '>') {
                            return new SQLToken("<=>", pos, 3, OP_NULL_SAFE_EQUALS);
                        }
                        reader.backForce();
                    }
                    return new SQLToken("<=", pos, 2, OP_LESS_OR_EQUALS);
                case '>':
                    return new SQLToken("<>", pos, 2, OP_LESS_OR_GREATER);
                }
                reader.backForce();
            }
            return new SQLToken("<", pos, 1, OP_LESS_THAN);
        case '>':
            if (reader.hasNext()) {
                c = reader.next();
                switch (c) {
                case '=':
                    return new SQLToken(">=", pos, 2, OP_GREATER_OR_EQUALS);
                case '>':
                    return new SQLToken(">>", pos, 2, OP_RIGHT_SHIFT);
                default:
                    reader.backForce();
                    return new SQLToken(">", pos, 1, OP_GREATER_THAN);
                }
            } else {
                return new SQLToken(">", pos, 1, OP_GREATER_THAN);
            }
        case '|':
            if (reader.hasNext()) {
                if (reader.next() == '|') {
                    return new SQLToken("||", pos, 2, OP_LOGICAL_OR);
                }
                reader.backForce();
            }
            return new SQLToken("|", pos, 1, OP_VERTICAL_BAR);
        case '&':
            if (reader.hasNext()) {
                if (reader.next() == '&') {
                    return new SQLToken("&&", pos, 2, OP_LOGICAL_AND);
                }
                reader.backForce();
            }
            return new SQLToken("&", pos, 1, OP_AMPERSAND);
        case '!':
            if (reader.hasNext()) {
                if (reader.next() == '=') {
                    return new SQLToken("!=", pos, 2, OP_NOT_EQUALS);
                }
                reader.backForce();
            }
            return new SQLToken("!", pos, 1, OP_EXCLAMATION);
        default:
            err("unrecognized char for operator: " + c);
            return null;
        }
    }

    private SQLToken matchPunctuation() {
        final int pos = reader.getCurrentPosition();
        if (!reader.hasNext()) return new SQLToken(null, pos, 0, EOF);
        char c = reader.next();
        switch (c) {
        case ';':
            return new SQLToken(";", pos, 1, PUNC_SEMICOLON);
        case ',':
            return new SQLToken(",", pos, 1, PUNC_COMMA);
        case '(':
            return new SQLToken("(", pos, 1, PUNC_LEFT_PAREN);
        case ')':
            return new SQLToken(")", pos, 1, PUNC_RIGHT_PAREN);
        case '[':
            return new SQLToken("[", pos, 1, PUNC_LEFT_BRACKET);
        case ']':
            return new SQLToken("]", pos, 1, PUNC_RIGHT_BRACKET);
        case '{':
            return new SQLToken("{", pos, 1, PUNC_LEFT_BRACE);
        case '}':
            return new SQLToken("}", pos, 1, PUNC_RIGHT_BRACE);
        case ':':
            return new SQLToken(":", pos, 1, PUNC_COLON);
        default:
            return new SQLToken(String.valueOf(c), pos, 1, PUNC_OTHER);
        }
    }

    private SQLToken matchSpecialChar() {
        final int pos = reader.getCurrentPosition();
        if (!reader.hasNext()) return new SQLToken(null, pos, 0, EOF);
        char c = reader.next();
        switch (c) {
        case '?':
            return new SQLToken("?", pos, 1, SC_QUESTION);
        default:
            err("unrecognized char for specialChar: " + c);
            return null;
        }
    }

    private static enum LiteralSpecial {
        LITERAL_SPECIAL_TRUE,
        LITERAL_SPECIAL_FALSE,
        LITERAL_SPECIAL_NULL
    }

    private static final Map<String, LiteralSpecial> specialLiteral = initSpecialLiteral();

    private static Map<String, LiteralSpecial> initSpecialLiteral() {
        Map<String, LiteralSpecial> map = new HashMap<String, SQLLexer.LiteralSpecial>(3);
        map.put("NULL", LiteralSpecial.LITERAL_SPECIAL_NULL);
        map.put("FALSE", LiteralSpecial.LITERAL_SPECIAL_FALSE);
        map.put("TRUE", LiteralSpecial.LITERAL_SPECIAL_TRUE);
        return map;
    }

    private SQLToken convertIdentifierIfspecial(SQLToken t) {
        if (!IDENTIFIER.equals(t.getType())) {
            return t;
        }
        String upperText = t.getText().toUpperCase();
        SQLTokenType kwt = Keywords.getInstance().getKeywordType(upperText);
        if (kwt != null) {
            return new SQLToken(upperText, t.getPos(), t.getOrgSize(), kwt);
        }
        LiteralSpecial ls = specialLiteral.get(upperText);
        if (ls != null) {
            switch (ls) {
            case LITERAL_SPECIAL_NULL:
                return new SQLToken("NULL", t.getPos(), t.getOrgSize(), LITERAL_NULL);
            case LITERAL_SPECIAL_FALSE:
                return new SQLToken("FALSE", t.getPos(), t.getOrgSize(), LITERAL_BOOL_FALSE);
            case LITERAL_SPECIAL_TRUE:
                return new SQLToken("TRUE", t.getPos(), t.getOrgSize(), LITERAL_BOOL_TRUE);
            }
        }
        return t;
    }

    public SQLToken next() {
        while (reader.hasNext()) {
            skipSeparator();
            SQLTokenType alt = makeDecision();
            switch (alt) {
            case IDENTIFIER:
                SQLToken t = matchIdentifier(false);
                return convertIdentifierIfspecial(t);
            case LITERAL_STRING:
                return matchLiteralString();
            case LITERAL_NUMBER:
                return convertIdentifierIfspecial(matchLiteralNumber());
            case LITERAL_HEXADECIMAL:
                return convertIdentifierIfspecial(matchLiteralHex());
            case LITERAL_BIT_FIELD:
                return convertIdentifierIfspecial(matchLiteralBit());
            case USER_DEFINED_VARIABLE:
                return matchUserDefVar();
            case SYSTEM_VARIABLE:
                return matchSysVar();
            case EOF:
                return new SQLToken(null, reader.getCurrentPosition(), 0, SQLTokenType.EOF);
            case OPERATOR:
                return matchOperator();
            case PUNCTUATION:
                return matchPunctuation();
            case SPECIAL_CHAR:
                return matchSpecialChar();
            default:
                err();
            }
        }
        return new SQLToken(null, reader.getCurrentPosition(), 0, SQLTokenType.EOF);
    }

    /**
     * skip white space or comment
     */
    private void skipSeparator() {
        if (!reader.hasNext()) return;
        while (true) {
            while (reader.hasNext() && Character.isWhitespace(reader.lookAhead(0))) {
                reader.skipForce(1);
            }
            if (reader.hasNext(3) && reader.isNext("--") && Character.isWhitespace(reader.lookAhead(2))) {
                // "-- " comment
                reader.skipForce(3);
                while (reader.hasNext() && reader.next() != '\n');
            } else if (reader.isNext("/*")) {
                // "/*...*/" comment
                reader.skipForce(2);
                int state = 0;
                loop1: while (reader.hasNext()) {
                    switch (state) {
                    case 0:
                        if (reader.next() == '*') {
                            state = 1;
                        }
                        break;
                    case 1:
                        switch (reader.next()) {
                        case '/':
                            state = 2;
                            break;
                        case '*':
                            state = 1;
                            break;
                        default:
                            state = 0;
                        }
                        break;
                    case 2:
                        break loop1;
                    }
                }
                if (state != 2) {
                    err();
                }
            } else if (reader.isNext("#")) {
                reader.next();
                while (reader.hasNext() && reader.next() != '\n');
            } else {
                break;
            }
        }
    }

    /**
     * always throw IllegalArgumentException
     * 
     * @throws IllegalArgumentException
     */
    private void err() throws IllegalArgumentException {
        err("Lexer err.");
    }

    /**
     * always throw IllegalArgumentException
     * 
     * @throws IllegalArgumentException
     */
    private void err(String msg) throws IllegalArgumentException {
        String errMsg =
                msg + ". position=" + reader.getCurrentPosition() + ", strLeft=" + new String(reader.getLeftChars());
        throw new IllegalArgumentException(errMsg);
    }

    /**
     * not <code>letter</code> , <code>digit</code> , <code>_</code> ,
     * <code>$</code>, <code>.</code>
     */
    private static boolean isEndCharacter(char c) {
        if (Character.isWhitespace(c) || Character.isISOControl(c)) return true;
        switch (c) {
        case '+':
        case '-':
        case '*':
        case '/':
        case '=':
        case '~':
        case '^':
        case '%':
        case '!':
        case '&':
        case '|':
        case '>':
        case '<':
        case ':':
        case '(':
        case ')':
        case '[':
        case ']':
        case '{':
        case '}':
        case '\\':
        case '\'':
        case '"':
        case ';':
        case '?':
        case ',':
        case '`':
        case '@':
        case '#':
            return true;
        default:
            return false;
        }
    }

    private static boolean isHexDigit(char c) {
        switch (c) {
        case '0':
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
        case '8':
        case '9':
        case 'a':
        case 'b':
        case 'c':
        case 'd':
        case 'e':
        case 'f':
        case 'A':
        case 'B':
        case 'C':
        case 'D':
        case 'E':
        case 'F':
            return true;
        default:
            return false;
        }
    }

    private static boolean isDigit(char c) {
        switch (c) {
        case '0':
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
        case '8':
        case '9':
            return true;
        default:
            return false;
        }
    }

    private boolean isSeparator() {
        if (!reader.hasNext()) return false;
        if (Character.isWhitespace(reader.lookAhead(0))) return true;
        if ((reader.isNext('/', 0) && reader.isNext('*', 1))
            || reader.isNext('#', 0)
            || (reader.hasNext(3) && reader.isNext("--") && Character.isWhitespace(reader.lookAhead(2)))) {
            return true;
        }
        return false;
    }

}
